package com.crawl.zhihu;

import com.crawl.Main;
import com.crawl.core.httpclient.AbstractHttpClient;
import com.crawl.core.httpclient.IHttpClient;
import com.crawl.core.util.*;
import com.crawl.proxy.ProxyHttpClient;
import com.crawl.zhihu.dao.ZhiHuDaoImp;
import com.crawl.zhihu.task.DetailListPageTask;
import com.crawl.zhihu.task.GeneralPageTask;
import org.apache.http.client.methods.HttpGet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.sql.Connection;
import java.sql.SQLException;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class ZhiHuHttpClient extends AbstractHttpClient implements IHttpClient{
    private static Logger logger = LoggerFactory.getLogger(Main.class);
    private volatile static ZhiHuHttpClient instance;
    /**
     * 统计用户数量
     */
    public static AtomicInteger parseUserCount = new AtomicInteger(0);
    private static long startTime = System.currentTimeMillis();
    public static volatile boolean isStop = false;

    public static ZhiHuHttpClient getInstance(){
        if (instance == null){
            synchronized (ZhiHuHttpClient.class){
                if (instance == null){
                    instance = new ZhiHuHttpClient();
                }
            }
        }
        return instance;
    }
    /**
     * 详情列表页下载线程池
     */
    private ThreadPoolExecutor detailListPageThreadPool;
    /**
     * request　header
     * 获取列表页时，必须带上
     */
    private static String authorization;
    private ZhiHuHttpClient() {
        initHttpClient();
        intiThreadPool();
    }
    /**
     * 初始化HttpClient
     */
    @Override
    public void initHttpClient() {
        if(Config.dbEnable){
            ZhiHuDaoImp.DBTablesInit();
        }
    }

    /**
     * 初始化线程池
     */
    private void intiThreadPool(){
        /**
         *
         * 创建一个corePoolSize为100，maxmumPoolSize为100。
         * 任务队列长度为2000的线程池，用于执行知乎详情列表页下载解析任务，其中poolSize大小可以通过配置文件修改
         * 当线程池中运行线程数量达到maxmumPoolSize为100，且队列长度达到2000。有新任务被添加进来，直接丢弃。
         * 这里调用的是SimpleThreadPoolExecutor,继承ThreadPoolExecutor，可以通过构造方法直接为线程池命名。
         * 这里继承的目的也是为了在输出日志中观察各个线程池运行状态。
         */
        detailListPageThreadPool = new SimpleThreadPoolExecutor(Config.downloadThreadSize,
                Config.downloadThreadSize,
                0L, TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<Runnable>(2000),
                new ThreadPoolExecutor.DiscardPolicy(),
                "detailListPageThreadPool");
        //开启一个新线程用于监视列表详情页测试线程池执行情况
        new Thread(new ThreadPoolMonitor(detailListPageThreadPool, "detailListPageThreadPool")).start();
    }

    /**
     * 开始爬取
     */
    @Override
    public void startCrawl() {
        //调用本来中的方法，初始化autorization字段
//        authorization = initAuthorization();
        //获取爬取入口用户token
        String startToken = Config.startUserToken;
        //根据token构建请求url
        String startUrl = String.format(Constants.USER_FOLLOWEES_URL, startToken, 0);
        //根据url创建一个GET请求
        HttpGet request = new HttpGet(startUrl);
        //设置autorizaton header
//        request.setHeader("authorization", "oauth " + ZhiHuHttpClient.getAuthorization());
        //创建一个DetailListPageTask，并添加至detailListPageThreadPool中执行
        detailListPageThreadPool.execute(new DetailListPageTask(request, Config.isProxy));
        manageHttpClient();
    }

    /**
     * 初始化authorization
     * @return
     */
    private String initAuthorization(){
        logger.info("初始化authoriztion中...");
        String content = null;
        //创建一个页面下载task
        GeneralPageTask generalPageTask = new GeneralPageTask(Config.startURL, true);
        //执行下载任务，直接调用run方法，
        generalPageTask.run();
        //获取下载成功的网页内容
        content = generalPageTask.getPage().getHtml();
        //创建一个正则表达式，获取authoriztion所在js文件地址的url
        Pattern pattern = Pattern.compile("https://static\\.zhihu\\.com/heifetz/main\\.app\\.([0-9]|[a-z])*\\.js");
        Matcher matcher = pattern.matcher(content);
        String jsSrc = null;
        if (matcher.find()){
            //解析出js文件url
            jsSrc = matcher.group(0);
        } else {
            throw new RuntimeException("not find javascript url");
        }
        String jsContent = null;
        //创建一个页面下载task，地址为刚刚解析出的js文件url
        GeneralPageTask jsPageTask = new GeneralPageTask(jsSrc, true);
        jsPageTask.run();
        //获取下载成功的js文件content
        jsContent = jsPageTask.getPage().getHtml();
        //创建一个正则表达式，解析出authoriztion字段值
        pattern = Pattern.compile("oauth (([0-9]|[a-z])+)");
        matcher = pattern.matcher(jsContent);
        if (matcher.find()){
            //获取authorization字段成功
            String authorization = matcher.group(1);
            logger.info("初始化authoriztion完成");
            return authorization;
        }
        throw new RuntimeException("not get authorization");
    }
    public static String getAuthorization(){
        return authorization;
    }
    /**
     * 管理知乎HttpClient
     * 关闭整个爬虫
     */
    public void manageHttpClient(){
        //每秒执行一次轮询，检测整个爬虫执行情况
        while (true) {
            /**
             * 下载网页数
             */
            long downloadPageCount = detailListPageThreadPool.getTaskCount();
            //下载网页数达到配置下载数，关闭detailListPageThreadPool线程池
            if (downloadPageCount >= Config.downloadPageCount &&
                    !detailListPageThreadPool.isShutdown()) {
                isStop = true;
                //设置ThreadPoolMonitor，isStopMonitor字段为true。关闭线程池监视类
                ThreadPoolMonitor.isStopMonitor = true;
                //关闭detailListPageThreadPool线程池
                detailListPageThreadPool.shutdown();
            }
            //判断detailListPageThreadPool线程池是否关闭完成，关闭完成后，在关闭数据库连接
            if(detailListPageThreadPool.isTerminated()){
                //关闭数据库连接
                Map<Thread, Connection> map = DetailListPageTask.getConnectionMap();
                for(Connection cn : map.values()){
                    try {
                        if (cn != null && !cn.isClosed()){
                            cn.close();
                        }
                    } catch (SQLException e) {
                        e.printStackTrace();
                    }
                }
                //关闭代理检测线程池
                ProxyHttpClient.getInstance().getProxyTestThreadPool().shutdownNow();
                //关闭代理下载页线程池
                ProxyHttpClient.getInstance().getProxyDownloadThreadPool().shutdownNow();
                break;
            }
            double costTime = (System.currentTimeMillis() - startTime) / 1000.0;//单位s
            logger.debug("抓取速率：" + parseUserCount.get() / costTime + "个/s");
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
    public ThreadPoolExecutor getDetailListPageThreadPool() {
        return detailListPageThreadPool;
    }

}
